
# data processing
import pandas as pd
import numpy as np

# load / save data
import codecs
import json

# save models
import pickle

# show data better
from pprint import pprint

# for getting text features
from sentence_transformers import SentenceTransformer

# for visualization
import plotly.express as px

# for dimension reduction and visualization
import umap

# for machine learning
from sklearn.ensemble import RandomForestClassifier

# load the this helper function from sklearn
from sklearn.model_selection import train_test_split

# for evaluate model
from sklearn.metrics import classification_report

# for re-sampling dataset
from imblearn.combine import SMOTEENN

print('* loaded all libraries')

* loaded all libraries


# as we use UTF-8 format to save the data
# we need to use `codecs` to open the file
raw_data = json.load(
    codecs.open('IO-8164.json', 'r', 'utf-8')
)
print("* loaded %s records" % (
    len(raw_data['data'])
))

* loaded 8164 records


pprint(raw_data['data'][10])

{'abstract': 'Background: Parathyroid carcinoma (PC) is an extremely rare '
             'malignant tumor with an incidence of about 6 new cases per 10 '
             'million inhabitants per year. While several papers have been '
             'published on treatments and outcomes of PC patients with '
             'loco-regional disease, little is known about the prognosis, '
             'treatment strategies, and prognostic factors of patients with '
             'distant metastasis. &lt;br/&gt;Material(s) and Method(s): We '
             'performed a systematic review and a pooled analysis of '
             'histopathologically confirmed PC cases published in literature '
             'using the following keywords: '
             '&quot;metastasis-metastatic-secondary nodes&quot; AND '
             '&quot;parathyroid carcinoma&quot;. Original case reports and '
             'case series reporting metastatic parathyroid carcinoma were '
             'included. Data from 58 articles were extracted in a piloted form '
             'by five reviewers on a shared database. &lt;br/&gt;Result(s): '
             'Seventy-nine patients with metastatic PC were identified between '
             '1898 and 2018. Ten (13%) patients had synchronous metastases, '
             'while metachronous metastases occurred in 43 (54%) patients. The '
             'remaining 26 patients developed metastatic disease concomitantly '
             'to local recurrence. Primary hyperparathyroidism guided the '
             'diagnosis of metastatic recurrence in 58 (73%) patients. Surgery '
             'was the main primary approach adopted, as it was performed in 43 '
             '(54%) patients. Twenty (25%) patients underwent systemic '
             'antineoplastic therapy, consisting of chemotherapy, '
             'immunotherapy, tyrosine kinase inhibitors, and hexestrol '
             'therapy. Bone resorption inhibitors had a limited efficacy in '
             'the long-term control of hypercalcemia. After a median follow-up '
             'of 37.5 months, 43 (55%) patients died, 22 (51%) due to the '
             'consequences of uncontrolled PHPT. The median overall survival '
             'was 36 months (range: 1-252). Surgery was associated with a '
             'better OS (HR 0.48, 95% CI 0.26-0.88), whereas bone metastases '
             'represented a negative prognostic factor (HR 2.7, 95% CI '
             '1.4-5.2). &lt;br/&gt;Conclusion(s): Metastatic PC has a '
             'relatively poor prognosis. The main goals of treatment are to '
             'counteract tumor growth and control hypercalcemia. Surgery of '
             'metastases is the best approach to achieve rapid control of PHPT '
             'and longer survival. Target therapies and immunotherapy deserve '
             'to be extensively tested in metastatic PC and strategies to '
             'better control hypercalcemia should be '
             'implemented.&lt;br/&gt;Copyright &amp;#xa9; 2022 Alberti, '
             'Smussi, Zamparini, Turla, Laini, Marchiselli, Grisanti, Bossi '
             'and Berruti.',
 'authors': 'Alberti A.; Smussi D.; Zamparini M.; Turla A.; Laini L.; '
            'Marchiselli C.; Grisanti S.; Bossi P.; Berruti A.',
 'date_created': 'Mon, 14 Nov 2022 15:03:44 GMT',
 'is_deleted': 'no',
 'is_pmid': False,
 'journal': 'Front. Oncol.',
 'meta': {'all_rct_ids': [],
          'pdfs': [],
          'pred': [{'is_rct': False, 'usr_fb': ''}],
          'pred_dict': {'ROBOTSEARCH_RCT': {'is_rct': False,
                                            'model': 'svm_cnn_ptyp',
                                            'preds': {'cnn': -0.6039859056472778,
                                                      'cnn_ptyp': -0.8455956135774115,
                                                      'ptyp': -0.24160970793013373,
                                                      'svm': -0.8258290711024487,
                                                      'svm_cnn': -0.667369686364534,
                                                      'svm_cnn_ptyp': -0.9089793942946676,
                                                      'svm_ptyp': -1.0674387790325826},
                                            'ptyp_rct': 0,
                                            'score': -0.9089793942946676,
                                            'threshold_type': 'balanced',
                                            'threshold_value': 2.1057231048584675}},
          'rct_id': '',
          'tags': []},
 'pid': '2019467181',
 'project_id': '60c0524c-3f41-11eb-b67e-000d3a9afec0',
 'pub_date': '26 Sep 2022',
 'rct_id': '',
 'seq_num': 8154,
 'short_name': 'Alberti A. et al 2022',
 'ss_ex': {'label': {}},
 'ss_pr': 'na',
 'ss_rs': 'na',
 'ss_st': 'a11',
 'study_type': '',
 'title': 'Treatment and outcome of metastatic parathyroid carcinoma: A '
          'systematic review and pooled analysis of published cases.',
 'year': '2022'}


# hold selected data as a list
rs = []

# a data dictionary for searching by pid(PMID or other id)
dd = {}

# check each record
for r in raw_data['data']:
    # save the information for building dataframe
    rs.append({
        'pid': r['pid'],
        'title': r['title'],
        'authors': r['authors'],
        'abstract': r['abstract'],
        'journal': r['journal'],
        'pub_date': r['pub_date'],
        # the label of decision
        'decision': r['ss_rs'],
    })
    
    # save to data dictionary
    dd[r['pid']] = r
    
df_all = pd.DataFrame(rs)
print('* loaded %s records' % (len(df_all)))

# output the first 5 records
df_all.head()

* loaded 8164 records


df_all['decision'].hist()

<AxesSubplot:>


# first, remove those na records
df = df_all[df_all['decision'] != 'na']
print('* left %s records after removing na' % (len(df)))

# second, convert the decision to y
def func_decision_to_y(r):
    if r['decision'] in ['f1', 'f3']:
        return 1
    else:
        return 0
    
df['y'] = df.apply(func_decision_to_y, axis=1)

# let's see how the updated dataframe looks like
df.head()

* left 6584 records after removing na

/Users/m210842/.local/lib/python3.7/site-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  if sys.path[0] == '':


print('* y=1: %s' % (len(df[df['y']==1])))
print('* y=0: %s' % (len(df[df['y']==0])))
df['y'].hist()

* y=1: 314
* y=0: 6270

<AxesSubplot:>

pip install -U sentence-transformers


# load model
model_name = 'multi-qa-MiniLM-L6-cos-v1'
model = SentenceTransformer(model_name)
print('* loaded model!')

* loaded model!


embeddings_title = model.encode(df.title.tolist())
print('* got title embeddings: ', embeddings_title.shape)

* got title embeddings:  (6584, 384)


embeddings_abs = model.encode(df.abstract.tolist())
print('* got abstract embeddings: ', embeddings_abs.shape)

* got abstract embeddings:  (6584, 384)


# create a dataset object for saving
dataset = {
    'embeddings_title': embeddings_title,
    'embeddings_abs': embeddings_abs,
    'y': df.y.tolist()
}

with open('dataset.pkl', 'wb') as f:
    pickle.dump(dataset, f)
    
print('* saved all data')

* saved all data


dataset = pickle.load(open('dataset.pkl', 'rb'))
embeddings_title = dataset['embeddings_title']
print('* got title embeddings: ', embeddings_title.shape)

embeddings_abs = dataset['embeddings_abs']
print('* got abstract embeddings: ', embeddings_abs.shape)

* got title embeddings:  (6584, 384)
* got abstract embeddings:  (6584, 384)


def func_get_rct(r):
    '''
    For a given r, guess whether it is a RCT.
    If it is RCT, return 1. Else return 0.
    
    We can use the result by ROBOTSEARCH_RCT as a referene
    '''
    # get pid for this record
    pid = r['pid']
    
    # search full data in data dictionary
    d = dd[pid]
    
    # get ROBOTSEARCH_RCT results
    if 'pred_dict' in d['meta']:
        is_rct = d['meta']['pred_dict']['ROBOTSEARCH_RCT']['is_rct']
    else:
        # in fact, the ROBOTSEARCH_RCT prediction may not be available.
        return 0
    
    if is_rct:
        return 1
    else:
        return 0

df['is_rct'] = df.apply(lambda r: func_get_rct(r), axis=1)

# Let's see how the is_RCT distributes.
# Obviously, most of the studies are not RCT.
df['is_rct'].hist()

/Users/m210842/.local/lib/python3.7/site-packages/ipykernel_launcher.py:26: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

<AxesSubplot:>


def func_get_phase(r):
    '''
    For a given r, get the phase information.
    
    As there are different stage, we need to convert the phase to a number:
    
    Not mentioned:   0
    Phase 1:         0.2
    Phase 2:         0.4
    Phase 3:         0.6
    Phase 4:         0.8
    Phase 5:         1
    
    This can be further improved to indicate small differences, such as 1b, 2b, etc.
    '''
    # let's just guess by the title
    # to avoid upper/lower case issue, convert to lower case 
    title = r['title'].lower()
    
    # let's check each phase
    # this part can be improved by NLP algorithms in future
    if 'phase v' in title or 'phase 5' in title:
        return 1
    if 'phase iv' in title or 'phase 4' in title:
        return 0.8
    if 'phase iii' in title or 'phase 3' in title:
        return 0.6
    elif 'phase ii' in title or 'phase 2' in title:
        return 0.4
    elif 'phase i' in title or 'phase 1' in title:
        return 0.2
    else:
        return 0

df['which_phase'] = df.apply(lambda r: func_get_phase(r), axis=1)

# Let's see how the phase distributes.
# Obviously, most of the studies do not show phase
# and there are a lot showing phase 2 (>1400)
# a lot showing phase 3 (>500)
df['which_phase'].hist()

/Users/m210842/.local/lib/python3.7/site-packages/ipykernel_launcher.py:35: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

<AxesSubplot:>


def func_get_srma(r):
    '''
    For given r, determine whether it is a SRMA study
    
    if it is a SRMA study, return 1. 
    else return 0.
    '''
    # let's just guess by the title
    # to avoid upper/lower case issue, convert to lower case 
    title = r['title'].lower()
    
    if 'systematic review' in title or 'meta-analysis' in title:
        return 1
    else:
        return 0
    
df['is_srma'] = df.apply(lambda r: func_get_srma(r), axis=1)

# Let's see how the SRMA study distributes.
# More than 1300 I guess
df['is_srma'].hist()

/Users/m210842/.local/lib/python3.7/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

<AxesSubplot:>


# for better visualizaion, create a temp dataframe dfv
dfv = df.copy()

# wrap long title
dfv['title_wrap'] = dfv.title.str.wrap(30)
dfv['title_wrap'] = dfv.title_wrap.apply(lambda x: x.replace('\n', '<br>'))
dfv.head()


dfv.groupby(by=['is_rct', 'y'])[['pid']].count()


dfv.groupby(by=['is_srma', 'y'])[['pid']].count()


dfv.groupby(by=['which_phase', 'y'])[['pid']].count()


# let's dimension reduction on title embeddings
embedding2d = umap.UMAP().fit_transform(
    embeddings_title
)

# bind to dataframe for vis
dfv['ebd_t2d_x'] = embedding2d[:, 0]
dfv['ebd_t2d_y'] = embedding2d[:, 1]


# make it ea
fig = px.scatter(
    dfv,
    x='ebd_t2d_x', 
    y='ebd_t2d_y',
    color='y',
    hover_data=[
        'title_wrap',
        'journal',
        'decision'
    ],
    height=600
)
# add hover template for complex vis
# fig.update_traces(hovertemplate="Decision: %{text}")

fig.show()


# let's dimension reduction on abs embeddings
embedding_abs_2d = umap.UMAP().fit_transform(
    embeddings_abs
)

# bind to dataframe for vis
dfv['ebd_a2d_x'] = embedding_abs_2d[:, 0]
dfv['ebd_a2d_y'] = embedding_abs_2d[:, 1]


# make it ea
fig = px.scatter(
    dfv,
    x='ebd_a2d_x', 
    y='ebd_a2d_y',
    color='y',
    hover_data=[
        'title_wrap',
        'journal',
        'decision'
    ],
    height=600
)
# add hover template for complex vis
# fig.update_traces(hovertemplate="Decision: %{text}")

fig.show()


# combine the dataset
X = np.concatenate([
    # text features
    embeddings_title, 
    embeddings_abs,
    
    # other features
    df[['which_phase', 'is_rct', 'is_srma']].to_numpy(),
],
    axis=1
)
y = df.y.to_numpy()
print('* combined feature vector:', X.shape)
print('* y:', y.shape)

* combined feature vector: (6584, 771)
* y: (6584,)


X_train, X_test, y_train, y_test = train_test_split(
    X, 
    y,
    test_size=0.2, 
    random_state=42
)
print('* X_train:', X_train.shape)
print('* X_test:', X_test.shape)
print('* y_train:', y_train.shape)
print('* y_test:', y_test.shape)

* X_train: (5267, 771)
* X_test: (1317, 771)
* y_train: (5267,)
* y_test: (1317,)


from collections import Counter

# let's try the SMOTEENN method 
from imblearn.combine import SMOTEENN

# create an instance of SMOTEENN
smote_enn = SMOTEENN(random_state=0)

# before resample
print("* original distribution:", sorted(Counter(y_train).items()))

# resample
X_train_resampled, y_train_resampled = smote_enn.fit_resample(X_train, y_train)
print("* resampled distribution:", sorted(Counter(y_train_resampled).items()))

* original distribution: [(0, 5006), (1, 261)]
* resampled distribution: [(0, 2480), (1, 4972)]


from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(max_depth=8, random_state=0)
print('* created a RFC model')

* created a RFC model


model.fit(X_train_resampled, y_train_resampled)
print('* trained model')

* trained model


from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

# get the prediction value
y_pred = model.predict(X_test)

print('Accuracy     : %.3f'%accuracy_score(y_test, y_pred))
print('Precision    : %.3f'%precision_score(y_test, y_pred))
print('Recall       : %.3f'%recall_score(y_test, y_pred))
print('F1-Score     : %.3f'%f1_score(y_test, y_pred))
print('\nClassification Report : ')
print(classification_report(y_test, y_pred))

Accuracy     : 0.798
Precision    : 0.141
Recall       : 0.792
F1-Score     : 0.240

Classification Report : 
              precision    recall  f1-score   support

           0       0.99      0.80      0.88      1264
           1       0.14      0.79      0.24        53

    accuracy                           0.80      1317
   macro avg       0.57      0.80      0.56      1317
weighted avg       0.96      0.80      0.86      1317

	pid	title	authors	abstract	journal	pub_date	decision
0	2019801890	Effects of Carbon Nanomaterials and Aloe vera ...	de Carvalho Lima E.N.; Barros Martins G.L.; Di...	Melanoma is an aggressive skin cancer that aff...	Pharmaceutics	October 2022	na
1	2015544162	Comorbidities and mortality rate in COVID-19 p...	Naimi A.; Yashmi I.; Jebeleh R.; Imani Mofrad ...	Introduction: The global pandemic of coronavir...	J. Clin. Lab. Anal.	May 2022	na
2	2018429700	MODERN BREAST CANCER THERAPY: FROM TAMOXIFEN T...	Shevchenko Ju.A.; Kuznetsova M.S.; Khristin A....	The purpose of the study was to conduct a syst...	Siberian J. Oncol.	2022	na
3	638894896	Atezolizumab plus Bevacizumab in Patients with...	Mao L.; Fang M.; Wei X.; Cao J.; Lin J.; Zhang...	PURPOSE: Anti-programmed cell death-1 monother...	Clin Cancer Res	01 Nov 2022	na
4	2020873780	Lifileucel, a Tumor-Infiltrating Lymphocyte Th...	Sarnaik A.A.; Hamid O.; Khushalani N.I.; Lewis...	PURPOSE Effective treatment options are limite...	J. Clin. Oncol.	20 Aug 2021	na

	pid	title	authors	abstract	journal	pub_date	decision
1347	2018048671	A Systematic Review and Meta-Analysis of Immun...	Wang Y.; Kong D.; Wang C.; Chen J.; Li J.; Liu...	Objective: We aimed to evaluate immune-related...	Technol. Cancer Res. Treat.	2020	e2
1348	2018071173	Evolving landscape of first-line combination t...	Lalani A.-K.A.; Heng D.Y.C.; Basappa N.S.; Woo...	Background: Renal cell carcinoma (RCC) is a co...	Ther. Adv. Med. Oncol.	2022	e2
1349	2013308385	Cardiotoxicity associated with immune checkpoi...	Rubio-Infante N.; Ramirez-Flores Y.A.; Castill...	Aims: This study aimed to estimate the inciden...	Eur. J. Heart Fail.	October 2021	e2
1350	2014514108	Sarcopenia's prognostic impact on patient...	Lee D.; Kim N.W.; Kim J.Y.; Lee J.H.; Noh J.H....	Background: Although sarcopenia has been repor...	J. Clin. Med.	November-2 2021	e2
1351	2017980445	Toripalimab Combined With Hepatic Arterial Inf...	Xu Y.-J.; Lai Z.-C.; He M.-K.; Bu X.-Y.; Chen ...	Purpose: Immunotherapy combined with chemother...	Technol. Cancer Res. Treat.	2021	e2

	pid	title	authors	abstract	journal	pub_date	decision	is_srma	title_wrap
1347	2018048671	A Systematic Review and Meta-Analysis of Immun...	Wang Y.; Kong D.; Wang C.; Chen J.; Li J.; Liu...	Objective: We aimed to evaluate immune-related...	Technol. Cancer Res. Treat.	2020	e2	1	A Systematic Review and Meta-<br>Analysis of I...
1348	2018071173	Evolving landscape of first-line combination t...	Lalani A.-K.A.; Heng D.Y.C.; Basappa N.S.; Woo...	Background: Renal cell carcinoma (RCC) is a co...	Ther. Adv. Med. Oncol.	2022	e2	1	Evolving landscape of first-<br>line combinati...
1349	2013308385	Cardiotoxicity associated with immune checkpoi...	Rubio-Infante N.; Ramirez-Flores Y.A.; Castill...	Aims: This study aimed to estimate the inciden...	Eur. J. Heart Fail.	October 2021	e2	1	Cardiotoxicity associated with<br>immune check...
1350	2014514108	Sarcopenia's prognostic impact on patient...	Lee D.; Kim N.W.; Kim J.Y.; Lee J.H.; Noh J.H....	Background: Although sarcopenia has been repor...	J. Clin. Med.	November-2 2021	e2	1	Sarcopenia's prognostic<br>impact on pati...
1351	2017980445	Toripalimab Combined With Hepatic Arterial Inf...	Xu Y.-J.; Lai Z.-C.; He M.-K.; Bu X.-Y.; Chen ...	Purpose: Immunotherapy combined with chemother...	Technol. Cancer Res. Treat.	2021	e2	0	Toripalimab Combined With<br>Hepatic Arterial ...

Abstract Screening¶

Load all libraries¶

Load Data¶

Check the data structure¶

Select and convert raw data to DataFrame¶

Data Pre-processing¶

Statistics on df¶

Add classification label¶

Check distribution¶

Get text features¶

Load models¶

Convert text to vector¶

Feature 1: Title Text¶

Feature 2: Abstract Text¶

Save embeddings¶

Load embeddings¶

Get other features¶

Feature 1: is RCT?¶

Feature 2: Which phase¶

Feature 3: is SRMA?¶

EDA (Optional)¶

Prepare EDA data¶

Explore the structure data¶

Explore the text data¶

Prepare dataset¶

Split train and test¶

Over- and Under-sampling¶

Build Model¶

Train Model¶

Evaluate Model¶

		pid
which_phase	y
0.0	0	4017
0.0	1	204
0.2	0	306
0.2	1	1
0.4	0	1400
0.4	1	24
0.6	0	546
0.6	1	85
0.8	0	1

		pid
is_rct	y
0	0	5661
0	1	281
1	0	609
1	1	33

		pid
is_srma	y
0	0	4983
0	1	173
1	0	1287
1	1	141